#!/usr/bin/env bash

export PATH="/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin"
export HOME=`cat /etc/passwd | grep "^\`whoami\`:" | awk -F: '{print $6}'`

# [ DEBUG ]
printf "\n\n[`date '+%F %H:%M:%S'`]" >> /tmp/.____post-deploy.log 2>&1
printf "\n${@}" >> /tmp/.____post-deploy.log 2>&1

# git push 完成之后传给 post-update 的参数数量与同批次推送的分支数量相同，
# 参数格式类似：refs/heads/s refs/heads/S

# [ 错误类型 ]
# err4 :目标端磁盘容量不足
# err5 :目标端权限不足
# err6 :目标端文件冲突
# err7 :目标端路径不存在

zPathOnHost=`dirname \`pwd\``

# 传输过程中IPv6 地址中的冒号以 '_' 进行了替换，此处需要还原
zMasterAddr=`echo $1 | awk -F@ '{print $2}' | sed 's/_/:/g'`
zMasterPort=`echo $1 | awk -F@ '{print $3}'`

zRepoID=`echo $1 | awk -F@ '{print $4}'`

# 传输过程中IPv6 地址中的冒号以 '_' 进行了替换，此处需要还原
zSelfIpStrAddr=`echo $1 | awk -F@ '{print $5}' | sed 's/_/:/g'`

# 本次布署的唯一身份标识
zDpID=`echo $1 | awk -F@ '{print $6}'`

zTimeStamp=`echo $1 | awk -F@ '{print $7}'`
zDpingSig=`echo $1 | awk -F@ '{print $8}'`

# 若用户想指定不同于项目创建时的运行路径，可使用此 alias 路径
zRepoAliasPath=`echo $1 | awk -F@ '{print $9}'`

zForceDpMark=`echo $1 | awk -F@ '{print $10}'`

zRepoOnLinePath="`dirname \`dirname \\\`dirname ${zPathOnHost}\\\`\``/`basename ${zPathOnHost}`"

# 当次布署的活动分支，仅用于所有动作完成后，进行清理
zServBranch="s@${zMasterAddr}@${zMasterPort}@${zRepoID}@${zSelfIpStrAddr}@${zDpID}@${zTimeStamp}@${zDpingSig}@${zRepoAliasPath}@${zForceDpMark}"
zShadowBranch="S@${zMasterAddr}@${zMasterPort}@${zRepoID}@${zSelfIpStrAddr}@${zDpID}@${zTimeStamp}@${zDpingSig}@${zRepoAliasPath}@${zForceDpMark}"

# 临时保存错误信息的位置
zErrLogPath="/tmp/.${zRepoID}_${zTimeStamp}_errlog"

# 预置为空
zOldMasterSig=

# 预置为空
zRecvContent=


# 'SN': 阶段性成功上报
# 'EN': 错误信息上报
zTcpReply() {
    exec 4<>/dev/tcp/${zMasterAddr}/${zMasterPort}
    printf "{\"repoID\":${zRepoID},\"opsID\":${1},\"hostAddr\":\"${zSelfIpStrAddr}\",\"dpID\":${zDpID},\"timeStamp\":${zTimeStamp},\"replyType\":\"${2}\",\"content\":\"${3}\"}">&4
    zRecvContent="`cat<&4`"  # bash tcp fd: 4
    exec 4<&-
    exec 4>&-
}

# 任何一环节失败，将调用此函数处理错误并回撤至原先的版本
zExitClean() {
    # 尝试回撤至原始版本
    # 此处只能尽力而为，无法进入项目路径，也无进一步措施可用
    cd $zPathOnHost
    if [[ 0 -eq $? ]]; then
        export GIT_DIR="${zPathOnHost}/.git"
        git stash
        git stash clear
        git branch master
        git checkout master
        git reset -q --hard $zOldMasterSig
    fi

    # 处理掉错误日志可能存在会与服务端 SQL 日志冲突的字符
    sed -i 's/[[:blank:]]\+/ /g' ${zErrLogPath}
    sed -i "s/\'/ /g" ${zErrLogPath}
    sed -i 's/"/|/g' ${zErrLogPath}
    sed -i "s/\n/;/g" ${zErrLogPath}

    # 反馈错误信息至服务端
    zTcpReply 8 "${1}" "${2}"

    # 清理分支
    git branch -D ${zServBranch}
    git branch -M ${zShadowBranch} "meta@${zMasterAddr}@${zMasterPort}@${zRepoID}@${zSelfIpStrAddr}"

    # 退出之前还原权限
    chmod 0755 ${zPathOnHost}/.git/hooks/post-update

    exit 255
}


zrun() {
    # 首先测试是否可与服务端正常通信
    zTcpReply 0 "" ""
    if [[ "!" != ${zRecvContent} ]]; then
        exit 255
    fi

    # 检测必要的路径下是存在权限异常的文件
    mkdir -p ${zRepoAliasPath} ${zRepoOnLinePath} ${zPathOnHost} ${zPathOnHost}_SHADOW
    chown `whoami` ${zRepoOnLinePath} ${zPathOnHost} ${zPathOnHost}_SHADOW 2>${zErrLogPath}
    if [[ 0 -ne $? ]]; then
        zExitClean "E5" "`cat ${zErrLogPath}`"
    fi

    if [[ "" != ${zRepoAliasPath} ]]; then
        chown `whoami` ${zRepoAliasPath}
        if [[ 0 -ne $? ]]; then
            zExitClean "E5" "`cat ${zErrLogPath}`"
        fi
    fi

    # 权限无误，清除可能被创建的空目录
    rmdir ${zRepoAliasPath} ${zRepoOnLinePath} ${zPathOnHost} ${zPathOnHost}_SHADOW

    ##################
    # 进入项目代码库 #
    ##################
    cd ${zPathOnHost} 2>${zErrLogPath}
    if [[ 0 -ne $? ]]; then
        zExitClean "E7" "cd $zPathOnHost: `cat ${zErrLogPath}`"
    fi

    # git 环境注册
    export GIT_DIR="${zPathOnHost}/.git"
    git branch master

    # 保留旧的版本号，在布署失败时自动回滚
    zOldMasterSig=`git log master -1 --format=%H`

    # ==== 检测是否存在重复布署动作 ====
    # 比正在进行的版本更新，kill 之，并清理其创建的分支
    # 比正在进行的版本更老，通常是主机自请布署触发的动作，其时间戳会被置为 -1，此时什么都不需要做
    # ----????---- 同一版本，返回错误，同时要检测父进程号[BUG: 筛选结果不准确]
    for x in `ps ax -o pid,ppid,cmd|fgrep "post-update"|fgrep "@${zRepoID}@"|fgrep -v "grep"|sed 's/[[:blank:]]/-/g'|grep -vE "${$}-"`
    do
        zExistTimeStamp=`echo ${x}|awk -F@ '{print $6}'`
        if [[ ${zTimeStamp} -gt ${zExistTimeStamp} ]]; then
            kill -9 `echo ${x}|grep -o '^[0-9]\+'`
            cd .git && rm `echo ${x}|grep -o 'refs/heads/.*'`
            cd ${zPathOnHost}
        elif [[ ${zTimeStamp} -lt ${zExistTimeStamp} ]]; then
            git branch -D ${zServBranch} ${zShadowBranch}
            exit 0
        # else
        #     zExitClean "E8" "duplicate deploy：[IP1] ${zSelfIpStrAddr} [IP2] `echo ${x}|awk -F@ '{print $5}'|sed 's/_/:/g'`"
        fi
    done

    # 当前 hook 执行过程中要去掉执行权限，防止以下的 git 操作触发 hook 无限循环
    chmod 0444 ${zPathOnHost}/.git/hooks/post-update

    # 清除可能存在的由于 git 崩溃残留的锁文件
    rm -f ${zPathOnHost}/.git/index.lock ${zPathOnHost}_SHADOW/.git/index.lock

    # 通知服务端已收到代码：上报阶段性成果
    zTcpReply 8 "S3" "" &

    # 还原代码到工作区
    git reset -q --hard ${zDpingSig}
    if [[ 0 -ne $? ]]; then
        if [[ "Y" == ${zForceDpMark} ]]; then
            \ls -a | grep -vE '^(\.|\.\.|\.git)$' | xargs rm -rf
        fi

        git stash
        git stash clear
        git reset -q --hard ${zDpingSig} 2>${zErrLogPath}
        if [[ 0 -ne $? ]]; then
            zExitClean "E6" "git reset(${zPathOnHost}): `cat ${zErrLogPath}`"
        fi
    fi

    # 取 master 分支最新版本号，用于校验布署结果
    zMasterSig=`git log master -1 --format=%H`

    # 1、检查两个分支 git log 是否一致
    # 2、检查是否存在文件不一致现象(忽略新产生的未被 git 跟踪的文件)
    if [[ "${zMasterSig}" != "${zDpingSig}" ]]; then
        zExitClean "E6" "code version inconsistent(git log)"
    elif [[ 0 -ne "`git status --short --untracked-files=no | wc -l`" ]]; then
        zExitClean "E6" "work area inconsistent(git status): `pwd`"
    fi

    # 创建项目路径软链接
    # 检测是否有路长冲突，若有，则抛出错误
    if [[ "Y" == ${zForceDpMark} ]]; then
        rm -rf ${zRepoOnLinePath}
        ln -sT ${zPathOnHost} ${zRepoOnLinePath} 2>${zErrLogPath}
        if [[ 0 -ne $? ]]; then
            zExitClean "E6" "`cat ${zErrLogPath}`"
        fi
    else
        if [[ 0 -eq `ls ${zRepoOnLinePath} | wc -l` ]]; then
            ln -sT ${zPathOnHost} ${zRepoOnLinePath} 2>${zErrLogPath}
            if [[ 0 -ne $? ]]; then
                zExitClean "E6" "`cat ${zErrLogPath}`"
            fi
        elif [[ ('l' != `ls -l ${zRepoOnLinePath} | grep -o '^l'`)
            || ((${zPathOnHost} != `readlink -q ${zRepoOnLinePath}`) && (`dirname \`dirname ${zPathOnHost}\``/`basename ${zPathOnHost}` != `readlink -q ${zRepoOnLinePath}`)) ]]; then
            zExitClean "E6" "路径冲突: ${zRepoOnLinePath}"
        fi
    fi

    # 若用户指定了 alias 路径，则创建同名软链接指向项目库
    # 检测路径是否有冲突，若有，则抛出错误
    if [[ "Y" == ${zForceDpMark} ]]; then
        rm -rf ${zRepoAliasPath}
        ln -sT ${zPathOnHost} ${zRepoAliasPath} 2>${zErrLogPath}
        if [[ 0 -ne $? ]]; then
            zExitClean "E6" "`cat ${zErrLogPath}`"
        fi
    else
        if [[ "" != ${zRepoAliasPath} ]]; then
            if [[ 0 -eq `ls ${zRepoAliasPath} | wc -l` ]]; then
                ln -sT ${zPathOnHost} ${zRepoAliasPath} 2>${zErrLogPath}
                if [[ 0 -ne $? ]]; then
                    zExitClean "E6" "`cat ${zErrLogPath}`"
                fi
            elif [[ ('l' != `ls -l ${zRepoAliasPath} | grep -o '^l'`)
                || ((${zPathOnHost} != `readlink -q ${zRepoAliasPath}`) && (`dirname \`dirname ${zPathOnHost}\``/`basename ${zPathOnHost}` != `readlink -q ${zRepoAliasPath}`)) ]]; then
                zExitClean "E6" "path confilict: ${zRepoAliasPath}"
            fi
        fi
    fi

    # 通知服务端已确认收到的内容准确无误：布署成功
    zTcpReply 8 "S4" "" &

    # 布署成功，还原权限
    chmod 0755 ${zPathOnHost}/.git/hooks/post-update

    # 等待确认服务端的全局结果
    # 若全局结果不是成功，则执行回退
    # 全局成功，服务端回复 "S"，失败回复 "F"，尚未确定最终结果回复 "W"
    sleep 5
    while :
    do
        zTcpReply 7 "" ""

        if [[ "S" == ${zRecvContent} ]]; then
            cd ${zRepoOnLinePath} && source ${zRepoOnLinePath}/____post-deploy.sh 2>/dev/null
            break
        elif [[ "F" == ${zRecvContent} ]]; then
            # 尝试回撤至原始版本
            # 此处只能尽力而为，若无法进入项目路径，也无进一步措施可用
            cd $zPathOnHost
            if [[ 0 -eq $? ]]; then
                export GIT_DIR="${zPathOnHost}/.git"
                git stash
                git stash clear
                git branch master
                git checkout master
                git reset -q --hard $zOldMasterSig
            fi

            git branch -D ${zServBranch}
            git branch -M ${zShadowBranch} "meta@${zMasterAddr}@${zMasterPort}@${zRepoID}@${zSelfIpStrAddr}"
            exit 255
        else
            # 继续等待
            sleep 2
        fi
    done

    # 删除已用完的分支
    git branch -D ${zServBranch}
    git branch -M ${zShadowBranch} "meta@${zMasterAddr}@${zMasterPort}@${zRepoID}@${zSelfIpStrAddr}"

    # 清理可能存在的旧版布署系统的遗留文件
    rm -rf ${zRepoOnLinePath}_SHADOW

    # git 仓库占用空间超过 200M 时，清理空间
    # 非必须动作，对执行周期要求不严格，故异常退出时，可不必执行此步
    cd ${zPathOnHost}
    if [[ 200 -lt `du -sm .git | grep -o '[0-9]\+'` ]]; then
        git reflog expire --expire=now --all
        git gc --aggressive --prune=all
    fi


    # ======== !!!! ========
    # 将不再兼容旧版布署创建的项目路径
    # 逐个项目升级时，依次重启相关服务
    # ======== !!!! ========
    cd ../../`basename ${zPathOnHost}`/.git
    if [[ 0 -eq $? ]]; then
        cd ..
        export GIT_DIR="`pwd`/.git"
        git branch master
        git checkout master
        git pull --force ${zPathOnHost}/.git ${zDpingSig}:${zDpingSig}
        git reset -q --hard ${zDpingSig}
    fi
}

# git push 需要等到 post-update 执行完毕后才会返回，因此在后台执行。
# tips：重定向 stdout 与 stderr 到 /dev/null 后，ssh 连接才会断开！
zrun >/dev/null 2>&1 &


######################
######################
######################
######################
cp ./hooks/post-update ./hooks/super-visor
sed -i "1,$((2 + ${LINENO}))d" ./hooks/super-visor
bash ./hooks/super-visor ${zMasterAddr} ${zMasterPort} ${zSelfIpStrAddr} ${zRepoID}>/dev/null 2>&1 &
exit 0
#!/usr/bin/env bash

zServAddr=$1
zServPort=$2
zSelfAddr=$3
zRepoID=$4

zInterval=5

zCpuTotalPrev=0
zCpuTotalCur=0

zCpuSpentPrev=0
zCpuSpentCur=0

zDiskIORDPrev=0
zDiskIORDCur=0

zDiskIOWRPrev=0
zDiskIOWRCur=0

zNetIORDPrev=0
zNetIORDCur=0

zNetIOWRPrev=0
zNetIOWRCur=0

zCpuTotal=0
zCpuSpent=0
zMemTotal=0
zMemSpent=0
zDiskUsage=0
zLoadAvg5=0

zDiskIORDSpent=0
zDiskIOWRSpent=0

zNetIORDSpent=0
zNetIOWRSpent=0

zReadData() {
    # CPU: total = user + system ＋nice + idle, spent = total - idle
    # /proc/stats
    # user   (1) Time spent in user mode.
    # nice   (2) Time spent in user mode with low priority (nice).
    # system (3) Time spent in system mode.
    # idle   (4) Time spent in the idle task.  This value should be USER_HZ times the second entry in the /proc/uptime pseudo-file.
    # ... ====> man 5 proc
    zCpuTotalCur=`cat /proc/stat | head -1 | awk -F' ' '{print $2,$3,$4,$5,$6,$7,$8,$9,$10}' | tr ' ' '+' | bc`
    zCpuTotal=$((${zCpuTotalCur} - ${zCpuTotalPrev}))

    # when unsigned int overflow...
    if [[ 0 -gt ${zCpuTotal} ]]; then
        return -1
    fi

    # spent = all - idle
    zCpuSpentCur=$((${zCpuTotalCur} - `cat /proc/stat | head -1 | awk -F' ' '{print $5}' | tr ' ' '+' | bc`))
    zCpuSpent=$((${zCpuSpentCur} - ${zCpuSpentPrev}))

    # when unsigned int overflow...
    if [[ 0 -gt ${zCpuSpent} ]]; then
        return -1
    fi

    # MEM: total = MemTotal
    # /proc/meminfo
    # MemTotal %lu
    #        Total usable RAM (i.e., physical RAM minus a few reserved bits and the kernel binary code).
    # MemFree %lu
    #        The sum of LowFree+HighFree.
    # Buffers %lu
    #        Relatively temporary storage for raw disk blocks that shouldn't get tremendously large (20MB or so).
    # Cached %lu
    #        In-memory cache for files read from the disk (the page cache).  Doesn't include SwapCached.
    zMemTotal=`cat /proc/meminfo | fgrep 'MemTotal' | grep -o '[0-9]\+'`  # Warning: memory hot-plug; Same as `free | fgrep -i 'mem' | awk -F' ' '{print $2}'`
    zMemSpent=`free | fgrep -i 'mem' | awk -F' ' '{print $3}'`  # different kernel is different, so use free utils...

    # DISK tps: /proc/diskstats
    #
    # ==== for disk ====
    # spent = <Field 1> - <Field 1 prev> + <Field 5> - <Field 5 prev>
    #
    # Field  1 -- # of reads completed
    #     This is the total number of reads completed successfully.
    # Field  2 -- # of reads merged, field 6 -- # of writes merged
    #     Reads and writes which are adjacent to each other may be merged for
    #     efficiency.  Thus two 4K reads may become one 8K read before it is
    #     ultimately handed to the disk, and so it will be counted (and queued)
    #     as only one I/O.  This field lets you know how often this was done.
    # Field  3 -- # of sectors read
    #     This is the total number of sectors read successfully.
    # Field  4 -- # of milliseconds spent reading
    #     This is the total number of milliseconds spent by all reads (as
    #     measured from __make_request() to end_that_request_last()).
    # Field  5 -- # of writes completed
    #     This is the total number of writes completed successfully.
    # Field  6 -- # of writes merged
    #     See the description of field 2.
    # Field  7 -- # of sectors written
    #     This is the total number of sectors written successfully.
    # Field  8 -- # of milliseconds spent writing
    #     This is the total number of milliseconds spent by all writes (as
    #     measured from __make_request() to end_that_request_last()).
    # Field  9 -- # of I/Os currently in progress
    #     The only field that should go to zero. Incremented as requests are
    #     given to appropriate struct request_queue and decremented as they finish.
    # Field 10 -- # of milliseconds spent doing I/Os
    #     This field increases so long as field 9 is nonzero.
    # Field 11 -- weighted # of milliseconds spent doing I/Os
    #     This field is incremented at each I/O start, I/O completion, I/O
    #     merge, or read of these stats by the number of I/Os in progress
    #     (field 9) times the number of milliseconds spent doing I/O since the
    #     last update of this field.  This can provide an easy measure of both
    #     I/O completion time and the backlog that may be accumulating.
    #
    # ==== for disk partition ====
    # spent = <Field 1> - <Field 1 prev> + <Field 3> - <Field 3 prev>
    #
    # Field  1 -- # of reads issued
    #     This is the total number of reads issued to this partition.
    # Field  2 -- # of sectors read
    #     This is the total number of sectors requested to be read from this
    #     partition.
    # Field  3 -- # of writes issued
    #     This is the total number of writes issued to this partition.
    # Field  4 -- # of sectors written
    #     This is the total number of sectors requested to be written to
    #     this partition.
    zDiskIORDCur=0;
    for x in `cat /proc/diskstats | fgrep -v loop | grep -v '[a-Z][0-9]' | awk -F' ' '{print $4,$5}'`
    do
        let zDiskIORDCur+=$x
    done
    zDiskIORDSpent=$((${zDiskIORDCur} - ${zDiskIORDPrev}))

    # when unsigned int overflow...
    if [[ 0 -gt ${zDiskIORDSpent} ]]; then
        return -1
    fi

    zDiskIOWRCur=0;
    for x in `cat /proc/diskstats | fgrep -v loop | grep -v '[a-Z][0-9]' | awk -F' ' '{print $8,$9}'`
    do
        let zDiskIOWRCur+=$x
    done
    zDiskIOWRSpent=$((${zDiskIOWRCur} - ${zDiskIOWRPrev}))

    # when unsigned int overflow...
    if [[ 0 -gt ${zDiskIOWRSpent} ]]; then
        return -1
    fi

    # NET tps: /proc/net/dev
    zNetIORDCur=0;
    for x in `cat /proc/net/dev | fgrep -v '|' | grep -vP '^\s*lo:' | sed 's/:/ /' | awk -F' ' '{print $3}'`
    do
        let zNetIORDCur+=$x
    done
    zNetIORDSpent=$((${zNetIORDCur} - ${zNetIORDPrev}));

    # when unsigned int overflow...
    if [[ 0 -gt ${zNetIORDSpent} ]]; then
        return -1
    fi

    zNetIOWRCur=0;
    for x in `cat /proc/net/dev | fgrep -v '|' | grep -vP '^\s*lo:' | sed 's/:/ /' | awk -F' ' '{print $11}'`
    do
        let zNetIOWRCur+=$x
    done
    zNetIOWRSpent=$((${zNetIOWRCur} - ${zNetIOWRPrev}));

    # when unsigned int overflow...
    if [[ 0 -gt ${zNetIOWRSpent} ]]; then
        return -1
    fi

    # DISK USAGE MAX: df
    zDiskUsage=0
    for x in `df | grep '^/dev' | awk -F' ' '{print $5}' | uniq | grep -o '[^%]\+'`
    do
        if [[ $i -lt $x ]]
        then
            zDiskUsage=$x
        fi
    done


    # LOADAVG5 /proc/loadavg
    zCPUNum=`cat /proc/cpuinfo | grep -c processor`  # Warning: CPU hot-plug
    zLoadAvg5=`echo "\`cat /proc/loadavg | awk -F' ' '{print $2}'\` * 10000 / ${zCPUNum}" | bc | grep -o '^[0-9]\+'`
}

# pre-exec once
zReadData

zCpuTotalPrev=${zCpuTotalCur}
zCpuSpentPrev=${zCpuSpentCur}
zDiskIORDPrev=${zDiskIORDCur}
zDiskIOWRPrev=${zDiskIORDCur}
zNetIORDPrev=${zNetIORDCur}
zNetIOWRPrev=${zNetIORDCur}

# clean old process...
if [[ 0 -lt `ps ax | grep "$0" | grep \`cat /tmp/.supervisor.pid\` | wc -l` ]]
then
    kill `cat /tmp/.supervisor.pid`
fi

echo $$ >/tmp/.supervisor.pid

# generate one udp socket
exec 7>&-
exec 7<&-
exec 7>/dev/udp/${zServAddr}/${zServPort}

# start...
while :
do
    sleep ${zInterval}

    zReadData

    if [[ 0 -gt $? ]]
    then
        continue
    fi

    # 开头的 '7'：是服务端的 udp 服务索引，请求记录监控信息
    # 不使用 echo，避免末尾自动追加 '\n'
    printf "7(${zRepoID},'${zSelfAddr}',`date +%s`,${zLoadAvg5},$((10000 * ${zCpuSpent} / ${zCpuTotal})),$((10000 * ${zMemSpent} / ${zMemTotal})),$((${zDiskIORDSpent} / ${zInterval})),$((${zDiskIOWRSpent} / ${zInterval})),$((${zNetIORDSpent} / ${zInterval})),$((${zNetIOWRSpent} / ${zInterval})),${zDiskUsage}),">&7

    zCpuTotalPrev=${zCpuTotalCur}
    zCpuSpentPrev=${zCpuSpentCur}
    zDiskIORDPrev=${zDiskIORDCur}
    zDiskIOWRPrev=${zDiskIORDCur}
    zNetIORDPrev=${zNetIORDCur}
    zNetIOWRPrev=${zNetIORDCur}
done
